{
    "cells": [
        {
            "cell_type": "code",
            "execution_count": 1,
            "id": "0c97b3f7",
            "metadata": {},
            "outputs": [],
            "source": [
                "# only run this if your have an editable install\n",
                "%load_ext autoreload\n",
                "%autoreload 2"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "fb762f06",
            "metadata": {},
            "source": [
                "## Prepare dataset\n",
                "\n",
                "prepare dataset so that it is off the form\n",
                "```\n",
                "question: list[str]\n",
                "answer: list[str]\n",
                "contexts: list[list[str]]\n",
                "ground_truths: list[list[str]]\n",
                "```"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 2,
            "id": "c76e1b8b",
            "metadata": {},
            "outputs": [
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "/home/jjmachan/jjmachan/explodinggradients/beir/code/beir/util.py:2: TqdmExperimentalWarning: Using `tqdm.autonotebook.tqdm` in notebook mode. Use `tqdm.tqdm` instead to force console mode (e.g. in jupyter console)\n",
                        "  from tqdm.autonotebook import tqdm\n"
                    ]
                }
            ],
            "source": [
                "from beir import util\n",
                "from beir.datasets.data_loader import GenericDataLoader\n",
                "\n",
                "dataset = \"fiqa\"\n",
                "url = (\n",
                "    \"https://public.ukp.informatik.tu-darmstadt.de/thakur/BEIR/datasets/{}.zip\".format(\n",
                "        dataset\n",
                "    )\n",
                ")\n",
                "data_path = util.download_and_unzip(url, \"datasets\")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 3,
            "id": "ad0b72cf",
            "metadata": {},
            "outputs": [],
            "source": [
                "import os\n",
                "import json\n",
                "import pandas as pd"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 4,
            "id": "f9ccb3fb",
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<div>\n",
                            "<style scoped>\n",
                            "    .dataframe tbody tr th:only-of-type {\n",
                            "        vertical-align: middle;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe tbody tr th {\n",
                            "        vertical-align: top;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe thead th {\n",
                            "        text-align: right;\n",
                            "    }\n",
                            "</style>\n",
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: right;\">\n",
                            "      <th></th>\n",
                            "      <th>_id</th>\n",
                            "      <th>title</th>\n",
                            "      <th>text</th>\n",
                            "      <th>metadata</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <th>0</th>\n",
                            "      <td>3</td>\n",
                            "      <td></td>\n",
                            "      <td>I'm not saying I don't like the idea of on-the...</td>\n",
                            "      <td>{}</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>1</th>\n",
                            "      <td>31</td>\n",
                            "      <td></td>\n",
                            "      <td>So nothing preventing false ratings besides ad...</td>\n",
                            "      <td>{}</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>2</th>\n",
                            "      <td>56</td>\n",
                            "      <td></td>\n",
                            "      <td>You can never use a health FSA for individual ...</td>\n",
                            "      <td>{}</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>3</th>\n",
                            "      <td>59</td>\n",
                            "      <td></td>\n",
                            "      <td>Samsung created the LCD and other flat screen ...</td>\n",
                            "      <td>{}</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>4</th>\n",
                            "      <td>63</td>\n",
                            "      <td></td>\n",
                            "      <td>Here are the SEC requirements: The federal sec...</td>\n",
                            "      <td>{}</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>...</th>\n",
                            "      <td>...</td>\n",
                            "      <td>...</td>\n",
                            "      <td>...</td>\n",
                            "      <td>...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>57633</th>\n",
                            "      <td>599946</td>\n",
                            "      <td></td>\n",
                            "      <td>&amp;gt;Well, first off, the roads are more than j...</td>\n",
                            "      <td>{}</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>57634</th>\n",
                            "      <td>599953</td>\n",
                            "      <td></td>\n",
                            "      <td>Yes they do. There are billions and billions s...</td>\n",
                            "      <td>{}</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>57635</th>\n",
                            "      <td>599966</td>\n",
                            "      <td></td>\n",
                            "      <td>&amp;gt;It's biggly sad you don't understand human...</td>\n",
                            "      <td>{}</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>57636</th>\n",
                            "      <td>599975</td>\n",
                            "      <td></td>\n",
                            "      <td>\"Did your CTO let a major group use \"\"admin/ad...</td>\n",
                            "      <td>{}</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>57637</th>\n",
                            "      <td>599987</td>\n",
                            "      <td></td>\n",
                            "      <td>Giving the government more control over the di...</td>\n",
                            "      <td>{}</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>\n",
                            "<p>57638 rows × 4 columns</p>\n",
                            "</div>"
                        ],
                        "text/plain": [
                            "          _id title                                               text  \\\n",
                            "0           3        I'm not saying I don't like the idea of on-the...   \n",
                            "1          31        So nothing preventing false ratings besides ad...   \n",
                            "2          56        You can never use a health FSA for individual ...   \n",
                            "3          59        Samsung created the LCD and other flat screen ...   \n",
                            "4          63        Here are the SEC requirements: The federal sec...   \n",
                            "...       ...   ...                                                ...   \n",
                            "57633  599946        &gt;Well, first off, the roads are more than j...   \n",
                            "57634  599953        Yes they do. There are billions and billions s...   \n",
                            "57635  599966        &gt;It's biggly sad you don't understand human...   \n",
                            "57636  599975        \"Did your CTO let a major group use \"\"admin/ad...   \n",
                            "57637  599987        Giving the government more control over the di...   \n",
                            "\n",
                            "      metadata  \n",
                            "0           {}  \n",
                            "1           {}  \n",
                            "2           {}  \n",
                            "3           {}  \n",
                            "4           {}  \n",
                            "...        ...  \n",
                            "57633       {}  \n",
                            "57634       {}  \n",
                            "57635       {}  \n",
                            "57636       {}  \n",
                            "57637       {}  \n",
                            "\n",
                            "[57638 rows x 4 columns]"
                        ]
                    },
                    "execution_count": 4,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "with open(os.path.join(data_path, \"corpus.jsonl\")) as f:\n",
                "    cs = [pd.Series(json.loads(l)) for l in f.readlines()]\n",
                "\n",
                "corpus_df = pd.DataFrame(cs)\n",
                "corpus_df"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 5,
            "id": "105d60d0",
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<div>\n",
                            "<style scoped>\n",
                            "    .dataframe tbody tr th:only-of-type {\n",
                            "        vertical-align: middle;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe tbody tr th {\n",
                            "        vertical-align: top;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe thead th {\n",
                            "        text-align: right;\n",
                            "    }\n",
                            "</style>\n",
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: right;\">\n",
                            "      <th></th>\n",
                            "      <th>corpus-id</th>\n",
                            "      <th>ground_truth</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <th>0</th>\n",
                            "      <td>3</td>\n",
                            "      <td>I'm not saying I don't like the idea of on-the...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>1</th>\n",
                            "      <td>31</td>\n",
                            "      <td>So nothing preventing false ratings besides ad...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>2</th>\n",
                            "      <td>56</td>\n",
                            "      <td>You can never use a health FSA for individual ...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>3</th>\n",
                            "      <td>59</td>\n",
                            "      <td>Samsung created the LCD and other flat screen ...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>4</th>\n",
                            "      <td>63</td>\n",
                            "      <td>Here are the SEC requirements: The federal sec...</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>\n",
                            "</div>"
                        ],
                        "text/plain": [
                            "   corpus-id                                       ground_truth\n",
                            "0          3  I'm not saying I don't like the idea of on-the...\n",
                            "1         31  So nothing preventing false ratings besides ad...\n",
                            "2         56  You can never use a health FSA for individual ...\n",
                            "3         59  Samsung created the LCD and other flat screen ...\n",
                            "4         63  Here are the SEC requirements: The federal sec..."
                        ]
                    },
                    "execution_count": 5,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "corpus_df = corpus_df.rename(columns={\"_id\": \"corpus-id\", \"text\": \"ground_truth\"})\n",
                "corpus_df = corpus_df.drop(columns=[\"title\", \"metadata\"])\n",
                "corpus_df[\"corpus-id\"] = corpus_df[\"corpus-id\"].astype(int)\n",
                "corpus_df.head()"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 6,
            "id": "933ad32e",
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<div>\n",
                            "<style scoped>\n",
                            "    .dataframe tbody tr th:only-of-type {\n",
                            "        vertical-align: middle;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe tbody tr th {\n",
                            "        vertical-align: top;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe thead th {\n",
                            "        text-align: right;\n",
                            "    }\n",
                            "</style>\n",
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: right;\">\n",
                            "      <th></th>\n",
                            "      <th>query-id</th>\n",
                            "      <th>question</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <th>0</th>\n",
                            "      <td>0</td>\n",
                            "      <td>What is considered a business expense on a bus...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>1</th>\n",
                            "      <td>4</td>\n",
                            "      <td>Business Expense - Car Insurance Deductible Fo...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>2</th>\n",
                            "      <td>5</td>\n",
                            "      <td>Starting a new online business</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>3</th>\n",
                            "      <td>6</td>\n",
                            "      <td>“Business day” and “due date” for bills</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>4</th>\n",
                            "      <td>7</td>\n",
                            "      <td>New business owner - How do taxes work for the...</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>\n",
                            "</div>"
                        ],
                        "text/plain": [
                            "   query-id                                           question\n",
                            "0         0  What is considered a business expense on a bus...\n",
                            "1         4  Business Expense - Car Insurance Deductible Fo...\n",
                            "2         5                     Starting a new online business\n",
                            "3         6            “Business day” and “due date” for bills\n",
                            "4         7  New business owner - How do taxes work for the..."
                        ]
                    },
                    "execution_count": 6,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "with open(os.path.join(data_path, \"queries.jsonl\")) as f:\n",
                "    qs = [pd.Series(json.loads(l)) for l in f.readlines()]\n",
                "\n",
                "queries_df = pd.DataFrame(qs)\n",
                "queries_df = queries_df.rename(columns={\"_id\": \"query-id\", \"text\": \"question\"})\n",
                "queries_df = queries_df.drop(columns=[\"metadata\"])\n",
                "queries_df[\"query-id\"] = queries_df[\"query-id\"].astype(int)\n",
                "queries_df.head()"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 7,
            "id": "e810fc4b",
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<div>\n",
                            "<style scoped>\n",
                            "    .dataframe tbody tr th:only-of-type {\n",
                            "        vertical-align: middle;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe tbody tr th {\n",
                            "        vertical-align: top;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe thead th {\n",
                            "        text-align: right;\n",
                            "    }\n",
                            "</style>\n",
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: right;\">\n",
                            "      <th></th>\n",
                            "      <th>query-id</th>\n",
                            "      <th>corpus-id</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <th>0</th>\n",
                            "      <td>1</td>\n",
                            "      <td>14255</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>1</th>\n",
                            "      <td>2</td>\n",
                            "      <td>308938</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>2</th>\n",
                            "      <td>3</td>\n",
                            "      <td>296717</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>3</th>\n",
                            "      <td>3</td>\n",
                            "      <td>100764</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>4</th>\n",
                            "      <td>3</td>\n",
                            "      <td>314352</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>\n",
                            "</div>"
                        ],
                        "text/plain": [
                            "   query-id  corpus-id\n",
                            "0         1      14255\n",
                            "1         2     308938\n",
                            "2         3     296717\n",
                            "3         3     100764\n",
                            "4         3     314352"
                        ]
                    },
                    "execution_count": 7,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "splits = [\"dev\", \"test\", \"train\"]\n",
                "split_df = {}\n",
                "for s in splits:\n",
                "    split_df[s] = pd.read_csv(os.path.join(data_path, f\"qrels/{s}.tsv\"), sep=\"\\t\").drop(\n",
                "        columns=[\"score\"]\n",
                "    )\n",
                "\n",
                "split_df[\"dev\"].head()"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 9,
            "id": "97ea4840",
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/plain": [
                            "500"
                        ]
                    },
                    "execution_count": 9,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "# how many unique querries\n",
                "len(set(split_df[\"dev\"][\"query-id\"]))"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 10,
            "id": "316d6991",
            "metadata": {},
            "outputs": [],
            "source": [
                "final_split_df = {}\n",
                "for split in split_df:\n",
                "    df = queries_df.merge(split_df[split], on=\"query-id\")\n",
                "    df = df.merge(corpus_df, on=\"corpus-id\")\n",
                "    df = df.drop(columns=[\"corpus-id\"])\n",
                "    grouped = df.groupby(\"query-id\").apply(\n",
                "        lambda x: pd.Series(\n",
                "            {\n",
                "                \"question\": x[\"question\"].sample().values[0],\n",
                "                \"ground_truths\": x[\"ground_truth\"].tolist(),\n",
                "            }\n",
                "        )\n",
                "    )\n",
                "\n",
                "    grouped = grouped.reset_index()\n",
                "    grouped = grouped.drop(columns=\"query-id\")\n",
                "    final_split_df[split] = grouped"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 11,
            "id": "942f8f40",
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<div>\n",
                            "<style scoped>\n",
                            "    .dataframe tbody tr th:only-of-type {\n",
                            "        vertical-align: middle;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe tbody tr th {\n",
                            "        vertical-align: top;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe thead th {\n",
                            "        text-align: right;\n",
                            "    }\n",
                            "</style>\n",
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: right;\">\n",
                            "      <th></th>\n",
                            "      <th>question</th>\n",
                            "      <th>ground_truths</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <th>0</th>\n",
                            "      <td>How to deposit a cheque issued to an associate...</td>\n",
                            "      <td>[Have the check reissued to the proper payee.,...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>1</th>\n",
                            "      <td>Can I send a money order from USPS as a business?</td>\n",
                            "      <td>[Sure you can.  You can fill in whatever you w...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>2</th>\n",
                            "      <td>1 EIN doing business under multiple business n...</td>\n",
                            "      <td>[You're confusing a lot of things here. Compan...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>3</th>\n",
                            "      <td>Applying for and receiving business credit</td>\n",
                            "      <td>[\"I'm afraid the great myth of limited liabili...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>4</th>\n",
                            "      <td>401k Transfer After Business Closure</td>\n",
                            "      <td>[You should probably consult an attorney. Howe...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>...</th>\n",
                            "      <td>...</td>\n",
                            "      <td>...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>643</th>\n",
                            "      <td>Closing a futures position</td>\n",
                            "      <td>[\"Assuming these are standardized and regulate...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>644</th>\n",
                            "      <td>Net loss not distributed by mutual funds to th...</td>\n",
                            "      <td>[When you invest (say $1000) in (say 100 share...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>645</th>\n",
                            "      <td>Pay off credit card debt or earn employer 401(...</td>\n",
                            "      <td>[A matching pension scheme is like free money....</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>646</th>\n",
                            "      <td>Short Term Capital Gains tax vs. IRA Withdrawa...</td>\n",
                            "      <td>[\"There is not a special rate for short-term c...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>647</th>\n",
                            "      <td>Am I required to have a lawyer create / overse...</td>\n",
                            "      <td>[\"This is not intended as legal advice, and on...</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>\n",
                            "<p>648 rows × 2 columns</p>\n",
                            "</div>"
                        ],
                        "text/plain": [
                            "                                              question  \\\n",
                            "0    How to deposit a cheque issued to an associate...   \n",
                            "1    Can I send a money order from USPS as a business?   \n",
                            "2    1 EIN doing business under multiple business n...   \n",
                            "3           Applying for and receiving business credit   \n",
                            "4                 401k Transfer After Business Closure   \n",
                            "..                                                 ...   \n",
                            "643                         Closing a futures position   \n",
                            "644  Net loss not distributed by mutual funds to th...   \n",
                            "645  Pay off credit card debt or earn employer 401(...   \n",
                            "646  Short Term Capital Gains tax vs. IRA Withdrawa...   \n",
                            "647  Am I required to have a lawyer create / overse...   \n",
                            "\n",
                            "                                         ground_truths  \n",
                            "0    [Have the check reissued to the proper payee.,...  \n",
                            "1    [Sure you can.  You can fill in whatever you w...  \n",
                            "2    [You're confusing a lot of things here. Compan...  \n",
                            "3    [\"I'm afraid the great myth of limited liabili...  \n",
                            "4    [You should probably consult an attorney. Howe...  \n",
                            "..                                                 ...  \n",
                            "643  [\"Assuming these are standardized and regulate...  \n",
                            "644  [When you invest (say $1000) in (say 100 share...  \n",
                            "645  [A matching pension scheme is like free money....  \n",
                            "646  [\"There is not a special rate for short-term c...  \n",
                            "647  [\"This is not intended as legal advice, and on...  \n",
                            "\n",
                            "[648 rows x 2 columns]"
                        ]
                    },
                    "execution_count": 11,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "final_split_df[\"test\"]"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 12,
            "id": "5b248ae0",
            "metadata": {},
            "outputs": [],
            "source": [
                "corpus_df = corpus_df.drop(columns=\"corpus-id\")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 13,
            "id": "1db36fb4",
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/html": [
                            "<div>\n",
                            "<style scoped>\n",
                            "    .dataframe tbody tr th:only-of-type {\n",
                            "        vertical-align: middle;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe tbody tr th {\n",
                            "        vertical-align: top;\n",
                            "    }\n",
                            "\n",
                            "    .dataframe thead th {\n",
                            "        text-align: right;\n",
                            "    }\n",
                            "</style>\n",
                            "<table border=\"1\" class=\"dataframe\">\n",
                            "  <thead>\n",
                            "    <tr style=\"text-align: right;\">\n",
                            "      <th></th>\n",
                            "      <th>doc</th>\n",
                            "    </tr>\n",
                            "  </thead>\n",
                            "  <tbody>\n",
                            "    <tr>\n",
                            "      <th>0</th>\n",
                            "      <td>I'm not saying I don't like the idea of on-the...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>1</th>\n",
                            "      <td>So nothing preventing false ratings besides ad...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>2</th>\n",
                            "      <td>You can never use a health FSA for individual ...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>3</th>\n",
                            "      <td>Samsung created the LCD and other flat screen ...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>4</th>\n",
                            "      <td>Here are the SEC requirements: The federal sec...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>...</th>\n",
                            "      <td>...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>57633</th>\n",
                            "      <td>&amp;gt;Well, first off, the roads are more than j...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>57634</th>\n",
                            "      <td>Yes they do. There are billions and billions s...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>57635</th>\n",
                            "      <td>&amp;gt;It's biggly sad you don't understand human...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>57636</th>\n",
                            "      <td>\"Did your CTO let a major group use \"\"admin/ad...</td>\n",
                            "    </tr>\n",
                            "    <tr>\n",
                            "      <th>57637</th>\n",
                            "      <td>Giving the government more control over the di...</td>\n",
                            "    </tr>\n",
                            "  </tbody>\n",
                            "</table>\n",
                            "<p>57638 rows × 1 columns</p>\n",
                            "</div>"
                        ],
                        "text/plain": [
                            "                                                     doc\n",
                            "0      I'm not saying I don't like the idea of on-the...\n",
                            "1      So nothing preventing false ratings besides ad...\n",
                            "2      You can never use a health FSA for individual ...\n",
                            "3      Samsung created the LCD and other flat screen ...\n",
                            "4      Here are the SEC requirements: The federal sec...\n",
                            "...                                                  ...\n",
                            "57633  &gt;Well, first off, the roads are more than j...\n",
                            "57634  Yes they do. There are billions and billions s...\n",
                            "57635  &gt;It's biggly sad you don't understand human...\n",
                            "57636  \"Did your CTO let a major group use \"\"admin/ad...\n",
                            "57637  Giving the government more control over the di...\n",
                            "\n",
                            "[57638 rows x 1 columns]"
                        ]
                    },
                    "execution_count": 13,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "corpus_df = corpus_df.rename(columns={\"ground_truth\": \"doc\"})\n",
                "corpus_df"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "69501463",
            "metadata": {},
            "source": [
                "### upload to hf space\n",
                "\n",
                "uploading to `explodinggradients/fiqa`. To make change however you have to clone it into the file path"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 14,
            "id": "8ba62313",
            "metadata": {},
            "outputs": [],
            "source": [
                "# change if new\n",
                "path_to_ds_repo = \"../../../../datasets/fiqa/\"\n",
                "import os\n",
                "\n",
                "assert os.path.exists(path_to_ds_repo), f\"{path_to_ds_repo} doesnot exist!\"\n",
                "\n",
                "for s in final_split_df:\n",
                "    final_split_df[s].to_csv(os.path.join(path_to_ds_repo, f\"{s}.csv\"), index=False)\n",
                "\n",
                "corpus_df.to_csv(os.path.join(path_to_ds_repo, \"corpus.csv\"), index=False)"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "8a661977",
            "metadata": {},
            "source": [
                "Now you have to go and commit the changes and upload the dataset"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 54,
            "id": "f298ff2f",
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Downloading and preparing dataset fiqa/main to /home/jjmachan/.cache/huggingface/datasets/fiqa/main/1.0.0/a3f6249910dbf03a46c718025ed536ea37e6975f8f4753704a0a9bf6d4a7d29f...\n"
                    ]
                },
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "d1d0d8711c5f4e0b8956f778071fbd61",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "Downloading data files:   0%|          | 0/3 [00:00<?, ?it/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "900affc613e947caa34f14930a97a8f1",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "Extracting data files:   0%|          | 0/3 [00:00<?, ?it/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "Generating train split: 0 examples [00:00, ? examples/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "Generating validation split: 0 examples [00:00, ? examples/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "Generating test split: 0 examples [00:00, ? examples/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Dataset fiqa downloaded and prepared to /home/jjmachan/.cache/huggingface/datasets/fiqa/main/1.0.0/a3f6249910dbf03a46c718025ed536ea37e6975f8f4753704a0a9bf6d4a7d29f. Subsequent calls will reuse this data.\n"
                    ]
                },
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "1311d01cce6044abbd4b7868f23b41f4",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "  0%|          | 0/3 [00:00<?, ?it/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "text/plain": [
                            "DatasetDict({\n",
                            "    train: Dataset({\n",
                            "        features: ['question', 'ground_truths'],\n",
                            "        num_rows: 5500\n",
                            "    })\n",
                            "    validation: Dataset({\n",
                            "        features: ['question', 'ground_truths'],\n",
                            "        num_rows: 500\n",
                            "    })\n",
                            "    test: Dataset({\n",
                            "        features: ['question', 'ground_truths'],\n",
                            "        num_rows: 648\n",
                            "    })\n",
                            "})"
                        ]
                    },
                    "execution_count": 54,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "from datasets import load_dataset\n",
                "\n",
                "fiqa = load_dataset(path_to_ds_repo, \"main\")\n",
                "fiqa"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "87923891",
            "metadata": {},
            "source": [
                "## Baseline\n",
                "\n",
                "Lets use llamaindex to create a baseline for our dataset"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 86,
            "id": "c9d0f1d4",
            "metadata": {},
            "outputs": [],
            "source": [
                "from llama_index import Document\n",
                "\n",
                "docs = []\n",
                "# ideally we should run this\n",
                "# for d in corpus_df[\"doc\"]:\n",
                "#     docs.append(Document(text=d))\n",
                "\n",
                "# for test\n",
                "for ds in final_split_df[\"test\"][\"ground_truths\"]:\n",
                "    docs.extend([Document(text=d) for d in ds])"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 87,
            "id": "19bbbe43",
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/plain": [
                            "1706"
                        ]
                    },
                    "execution_count": 87,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "len(docs)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 88,
            "id": "fe678d22",
            "metadata": {},
            "outputs": [],
            "source": [
                "# make nodes\n",
                "from llama_index.node_parser import SimpleNodeParser\n",
                "from langchain.text_splitter import TokenTextSplitter\n",
                "\n",
                "spliter = TokenTextSplitter(chunk_size=100, chunk_overlap=50)\n",
                "\n",
                "parser = SimpleNodeParser(text_splitter=spliter)\n",
                "\n",
                "nodes = parser.get_nodes_from_documents(documents=docs)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 89,
            "id": "3bf6c119",
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/plain": [
                            "Node(text='Have the check reissued to the proper payee.', doc_id='0d5a55b8-8931-4686-afc7-2351ae917c1a', embedding=None, doc_hash='fa12bca32af448285826c121ac50184bb97eb0d581fa487bfcd761f871561af1', extra_info=None, node_info=None, relationships={<DocumentRelationship.SOURCE: '1'>: '2d0d59b6-ba7d-4c91-8ec3-fcb5255883c8'})"
                        ]
                    },
                    "execution_count": 89,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "nodes[0]"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 90,
            "id": "2c0b1894",
            "metadata": {},
            "outputs": [],
            "source": [
                "from llama_index import GPTVectorStoreIndex, MockEmbedding\n",
                "from langchain.embeddings.huggingface import HuggingFaceEmbeddings\n",
                "from llama_index import LangchainEmbedding, ServiceContext, StorageContext\n",
                "\n",
                "# load in HF embedding model from langchain\n",
                "embed_model = LangchainEmbedding(HuggingFaceEmbeddings())\n",
                "hf_sc = ServiceContext.from_defaults(embed_model=embed_model)\n",
                "\n",
                "# mock embeddings\n",
                "embed_model = MockEmbedding(embed_dim=1536)\n",
                "mock = ServiceContext.from_defaults(embed_model=embed_model)\n",
                "\n",
                "# openai embeddings\n",
                "openai_sc = ServiceContext.from_defaults()"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "9a97ca3e",
            "metadata": {},
            "source": [
                "Now we are going to create the index but there is a lot of problems with how llama_index does this\n",
                "\n",
                "1. No batching or async for computing embeddings\n",
                "2. No progress bar to show how much has been embedded\n",
                "3. I need to see how much this embeddings is going to cost me"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 92,
            "id": "66be5435",
            "metadata": {},
            "outputs": [],
            "source": [
                "# create index\n",
                "index = GPTVectorStoreIndex.from_documents(\n",
                "    documents=docs,\n",
                "    service_context=openai_sc,\n",
                ")\n",
                "\n",
                "# query with embed_model specified\n",
                "qe = index.as_query_engine(mode=\"embedding\", verbose=True, service_context=openai_sc)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 93,
            "id": "4b3d899a",
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "question:  How to deposit a cheque issued to an associate in my business into my business account?\n",
                        "answer:  \n",
                        "The only way to deposit a cheque issued to an associate in your business into your business account is to open a business account with the bank. This requires a state-issued \"dba\" certificate from the county clerk's office as well as an Employer ID Number (EIN) issued by the IRS. The cheapest business banking account typically costs $15/month.\n"
                    ]
                }
            ],
            "source": [
                "q = final_split_df[\"test\"][\"question\"][0]\n",
                "print(\"question: \", q)\n",
                "\n",
                "r = qe.query(q)\n",
                "print(\"answer: \", r.response)"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 45,
            "id": "5928553b",
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "[\"Just have the associate sign the back and then deposit it.  It's called a third party cheque and is perfectly legal.  I wouldn't be surprised if it has a longer hold period and, as always, you don't get the money if the cheque doesn't clear. Now, you may have problems if it's a large amount or you're not very well known at the bank.  In that case you can have the associate go to the bank and endorse it in front of the teller with some ID.  You don't even technically have to be there.  Anybody can deposit money to your account if they have the account number. He could also just deposit it in his account and write a cheque to the business.\", '\"I have checked with Bank of America, and they say the ONLY way to cash (or deposit, or otherwise get access to the funds represented by a check made out to my business) is to open a business account. They tell me this is a Federal regulation, and every bank will say the same thing.  To do this, I need a state-issued \"\"dba\"\" certificate (from the county clerk\\'s office) as well as an Employer ID Number (EIN) issued by the IRS. AND their CHEAPEST business banking account costs $15 / month. I think I can go to the bank that the check is drawn upon, and they will cash it, assuming I have documentation showing that I am the sole proprietor. But I\\'m not sure.... What a racket!!\"']\n"
                    ]
                }
            ],
            "source": [
                "print([n.node.text for n in r.source_nodes])"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 99,
            "id": "be64248e",
            "metadata": {},
            "outputs": [],
            "source": [
                "# save the index\n",
                "index.storage_context.persist(persist_dir=\"./storage\")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 61,
            "id": "e47582d3",
            "metadata": {},
            "outputs": [],
            "source": [
                "# load the index\n",
                "from llama_index import StorageContext, load_index_from_storage, ServiceContext\n",
                "\n",
                "# rebuild storage context\n",
                "storage_context = StorageContext.from_defaults(persist_dir=\"./storage\")\n",
                "\n",
                "# load index\n",
                "index = load_index_from_storage(storage_context)\n",
                "\n",
                "# query with embed_model specified\n",
                "qe = index.as_query_engine(\n",
                "    mode=\"embedding\", verbose=True, service_context=openai_sc, use_async=False\n",
                ")"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 94,
            "id": "9816be1b",
            "metadata": {},
            "outputs": [],
            "source": [
                "from llama_index import (\n",
                "    GPTVectorStoreIndex,\n",
                "    ResponseSynthesizer,\n",
                ")\n",
                "from llama_index.retrievers import VectorIndexRetriever\n",
                "from llama_index.query_engine import RetrieverQueryEngine\n",
                "from llama_index.indices.postprocessor import SimilarityPostprocessor\n",
                "\n",
                "# configure retriever\n",
                "retriever = VectorIndexRetriever(\n",
                "    index=index,\n",
                "    similarity_top_k=3,\n",
                ")\n",
                "\n",
                "# configure response synthesizer\n",
                "response_synthesizer = ResponseSynthesizer.from_args(\n",
                "    node_postprocessors=[SimilarityPostprocessor(similarity_cutoff=0.7)]\n",
                ")\n",
                "\n",
                "# assemble query engine\n",
                "qe = RetrieverQueryEngine(\n",
                "    retriever=retriever,\n",
                "    response_synthesizer=response_synthesizer,\n",
                ")"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "dfdfa844",
            "metadata": {},
            "source": [
                "## Inference and uploading to huggingface"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 95,
            "id": "930ac4c3",
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "text/plain": [
                            "Dataset({\n",
                            "    features: ['question', 'ground_truths'],\n",
                            "    num_rows: 648\n",
                            "})"
                        ]
                    },
                    "execution_count": 95,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "test_ds = fiqa[\"test\"]\n",
                "test_ds"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 96,
            "id": "1ab58b36",
            "metadata": {},
            "outputs": [],
            "source": [
                "def generate_response(row):\n",
                "    r = qe.query(row[\"question\"])\n",
                "    row[\"answer\"] = r.response\n",
                "    row[\"contexts\"] = [sn.node.text for sn in r.source_nodes]\n",
                "\n",
                "    return row\n",
                "\n",
                "\n",
                "# generate_response(test_ds[0])"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 97,
            "id": "76037e00",
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "Map:   0%|          | 0/30 [00:00<?, ? examples/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "text/plain": [
                            "Dataset({\n",
                            "    features: ['question', 'ground_truths', 'answer', 'contexts'],\n",
                            "    num_rows: 30\n",
                            "})"
                        ]
                    },
                    "execution_count": 97,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "gen_ds = test_ds.select(range(30)).map(generate_response)\n",
                "gen_ds"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 66,
            "id": "503a568d",
            "metadata": {},
            "outputs": [
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "ef92f4ac29a14662af1237ad9c4fa7a5",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "Creating CSV from Arrow format:   0%|          | 0/1 [00:00<?, ?ba/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "text/plain": [
                            "149647"
                        ]
                    },
                    "execution_count": 66,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "path_to_ds_repo = \"../../../../datasets/fiqa/\"\n",
                "import os\n",
                "\n",
                "assert os.path.exists(path_to_ds_repo), f\"{path_to_ds_repo} doesnot exist!\"\n",
                "gen_ds.to_csv(os.path.join(path_to_ds_repo, \"baseline.csv\"))"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "216b2546",
            "metadata": {},
            "source": [
                "Now commit the repo and push changes"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 84,
            "id": "388af118",
            "metadata": {},
            "outputs": [
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "Downloading and preparing dataset fiqa/ragas_eval to /home/jjmachan/.cache/huggingface/datasets/fiqa/ragas_eval/1.0.0/953cfddc4a440cf2e290172be2563e5b51a953f2e4266940fc2b311e135cea69...\n"
                    ]
                },
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "dc97b56810ad45058d9412d93f26b65b",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "Downloading data files:   0%|          | 0/1 [00:00<?, ?it/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "2dbc958775e54546aa808057d717fb58",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "Extracting data files:   0%|          | 0/1 [00:00<?, ?it/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "Generating baseline split: 0 examples [00:00, ? examples/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "name": "stdout",
                    "output_type": "stream",
                    "text": [
                        "../../../../datasets/fiqa/baseline.csv\n",
                        "Dataset fiqa downloaded and prepared to /home/jjmachan/.cache/huggingface/datasets/fiqa/ragas_eval/1.0.0/953cfddc4a440cf2e290172be2563e5b51a953f2e4266940fc2b311e135cea69. Subsequent calls will reuse this data.\n"
                    ]
                },
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "ebb2d7b884474e369e1c11957f64a450",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "  0%|          | 0/1 [00:00<?, ?it/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "data": {
                        "text/plain": [
                            "DatasetDict({\n",
                            "    baseline: Dataset({\n",
                            "        features: ['question', 'ground_truths', 'answer', 'contexts'],\n",
                            "        num_rows: 30\n",
                            "    })\n",
                            "})"
                        ]
                    },
                    "execution_count": 84,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "load_dataset(path_to_ds_repo, \"ragas_eval\")"
            ]
        },
        {
            "cell_type": "markdown",
            "id": "97909bcf",
            "metadata": {},
            "source": [
                "## Evaluate with Ragas"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": 98,
            "id": "e4eb2a37",
            "metadata": {},
            "outputs": [
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "/home/jjmachan/miniconda3/envs/bench/lib/python3.10/site-packages/transformers/models/t5/tokenization_t5_fast.py:155: FutureWarning: This tokenizer was incorrectly instantiated with a model max length of 512 which will be corrected in Transformers v5.\n",
                        "For now, this behavior is kept to avoid breaking backwards compatibility when padding/encoding with `truncation is True`.\n",
                        "- Be aware that you SHOULD NOT rely on t5-base automatically truncating your input to 512 when padding/encoding.\n",
                        "- If you want to encode/pad to sequences longer than 512 you can either instantiate this tokenizer with `model_max_length` or pass `max_length` when encoding/padding.\n",
                        "- To avoid this warning, please instantiate this tokenizer with `model_max_length` set to your preferred value.\n",
                        "  warnings.warn(\n",
                        "100%|█████████████████████████████████████████████████████████████| 2/2 [01:15<00:00, 37.90s/it]\n"
                    ]
                },
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "Map:   0%|          | 0/30 [00:00<?, ? examples/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "100%|█████████████████████████████████████████████████████████████| 1/1 [00:04<00:00,  4.57s/it]\n"
                    ]
                },
                {
                    "data": {
                        "application/vnd.jupyter.widget-view+json": {
                            "model_id": "",
                            "version_major": 2,
                            "version_minor": 0
                        },
                        "text/plain": [
                            "Map:   0%|          | 0/30 [00:00<?, ? examples/s]"
                        ]
                    },
                    "metadata": {},
                    "output_type": "display_data"
                },
                {
                    "name": "stderr",
                    "output_type": "stream",
                    "text": [
                        "100%|█████████████████████████████████████████████████████████████| 3/3 [03:06<00:00, 62.05s/it]\n"
                    ]
                },
                {
                    "data": {
                        "text/plain": [
                            "{'NLI_score': 0.8655555555555556, 'answer_relevancy': 0.8737666666666667, 'context_ relevancy': 0.8181444444444443, 'ragas_score': 0.8517704492684051}"
                        ]
                    },
                    "execution_count": 98,
                    "metadata": {},
                    "output_type": "execute_result"
                }
            ],
            "source": [
                "from ragas.metrics import faithfulness, answer_relevancy, context_relevancy\n",
                "from ragas import evaluate\n",
                "\n",
                "evaluate(gen_ds, metrics=[faithfulness, answer_relevancy, context_relevancy])"
            ]
        },
        {
            "cell_type": "code",
            "execution_count": null,
            "id": "997df8cc",
            "metadata": {},
            "outputs": [],
            "source": []
        }
    ],
    "metadata": {
        "kernelspec": {
            "display_name": "Python 3 (ipykernel)",
            "language": "python",
            "name": "python3"
        },
        "language_info": {
            "codemirror_mode": {
                "name": "ipython",
                "version": 3
            },
            "file_extension": ".py",
            "mimetype": "text/x-python",
            "name": "python",
            "nbconvert_exporter": "python",
            "pygments_lexer": "ipython3",
            "version": "3.10.11"
        }
    },
    "nbformat": 4,
    "nbformat_minor": 5
}
